{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IOPS AI Challenge Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from typing import Final\n",
    "from collections.abc import Callable\n",
    "import matplotlib.pyplot as plt\n",
    "from config import data_raw_folder, data_processed_folder\n",
    "from timeeval import Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = (20, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking for source datasets in /home/projects/akita/data/benchmark-data/data-raw/IOPS AI Challenge/KPI Anomaly Detection Final Data Set and\n",
      "saving processed datasets in /home/projects/akita/data/benchmark-data/data-processed\n"
     ]
    }
   ],
   "source": [
    "dataset_collection_name = \"IOPS\"\n",
    "source_folder = os.path.join(data_raw_folder, \"IOPS AI Challenge/KPI Anomaly Detection Final Data Set\")\n",
    "target_folder = data_processed_folder\n",
    "\n",
    "from pathlib import Path\n",
    "print(f\"Looking for source datasets in {Path(source_folder).absolute()} and\\nsaving processed datasets in {Path(target_folder).absolute()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(df: pd.DataFrame, kpi: str) -> pd.DataFrame:\n",
    "    df = df[df[\"KPI ID\"] == kpi].copy()\n",
    "    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], unit=\"s\")\n",
    "    df = df[[\"timestamp\", \"value\", \"label\"]]\n",
    "    df.columns = [\"timestamp\", \"value\", \"is_anomaly\"]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Directories /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS already exist\n"
     ]
    }
   ],
   "source": [
    "train_type = \"supervised\"\n",
    "train_is_normal = False\n",
    "input_type = \"univariate\"\n",
    "datetime_index = True\n",
    "dataset_type = \"real\"\n",
    "\n",
    "# create target directory\n",
    "dataset_subfolder = os.path.join(input_type, dataset_collection_name)\n",
    "target_subfolder = os.path.join(target_folder, dataset_subfolder)\n",
    "try:\n",
    "    os.makedirs(target_subfolder)\n",
    "    print(f\"Created directories {target_subfolder}\")\n",
    "except FileExistsError:\n",
    "    print(f\"Directories {target_subfolder} already exist\")\n",
    "    pass\n",
    "\n",
    "dm = Datasets(target_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading source datasets...\n",
      "...done.\n",
      "\n",
      "Starting processing...\n",
      "Processed KPI da10a69f-d836-3baa-ad40-3e548ecf1fbd -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-da10a69f-d836-3baa-ad40-3e548ecf1fbd.train.csv\n",
      "Processed KPI da10a69f-d836-3baa-ad40-3e548ecf1fbd -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-da10a69f-d836-3baa-ad40-3e548ecf1fbd.test.csv\n",
      "Processed KPI e0747cad-8dc8-38a9-a9ab-855b61f5551d -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-e0747cad-8dc8-38a9-a9ab-855b61f5551d.train.csv\n",
      "Processed KPI e0747cad-8dc8-38a9-a9ab-855b61f5551d -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-e0747cad-8dc8-38a9-a9ab-855b61f5551d.test.csv\n",
      "Processed KPI ab216663-dcc2-3a24-b1ee-2c3e550e06c9 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-ab216663-dcc2-3a24-b1ee-2c3e550e06c9.train.csv\n",
      "Processed KPI ab216663-dcc2-3a24-b1ee-2c3e550e06c9 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-ab216663-dcc2-3a24-b1ee-2c3e550e06c9.test.csv\n",
      "Processed KPI 54350a12-7a9d-3ca8-b81f-f886b9d156fd -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-54350a12-7a9d-3ca8-b81f-f886b9d156fd.train.csv\n",
      "Processed KPI 54350a12-7a9d-3ca8-b81f-f886b9d156fd -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-54350a12-7a9d-3ca8-b81f-f886b9d156fd.test.csv\n",
      "Processed KPI a8c06b47-cc41-3738-9110-12df0ee4c721 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-a8c06b47-cc41-3738-9110-12df0ee4c721.train.csv\n",
      "Processed KPI a8c06b47-cc41-3738-9110-12df0ee4c721 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-a8c06b47-cc41-3738-9110-12df0ee4c721.test.csv\n",
      "Processed KPI 0efb375b-b902-3661-ab23-9a0bb799f4e3 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-0efb375b-b902-3661-ab23-9a0bb799f4e3.train.csv\n",
      "Processed KPI 0efb375b-b902-3661-ab23-9a0bb799f4e3 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-0efb375b-b902-3661-ab23-9a0bb799f4e3.test.csv\n",
      "Processed KPI c02607e8-7399-3dde-9d28-8a8da5e5d251 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-c02607e8-7399-3dde-9d28-8a8da5e5d251.train.csv\n",
      "Processed KPI c02607e8-7399-3dde-9d28-8a8da5e5d251 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-c02607e8-7399-3dde-9d28-8a8da5e5d251.test.csv\n",
      "Processed KPI 301c70d8-1630-35ac-8f96-bc1b6f4359ea -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-301c70d8-1630-35ac-8f96-bc1b6f4359ea.train.csv\n",
      "Processed KPI 301c70d8-1630-35ac-8f96-bc1b6f4359ea -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-301c70d8-1630-35ac-8f96-bc1b6f4359ea.test.csv\n",
      "Processed KPI 7103fa0f-cac4-314f-addc-866190247439 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-7103fa0f-cac4-314f-addc-866190247439.train.csv\n",
      "Processed KPI 7103fa0f-cac4-314f-addc-866190247439 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-7103fa0f-cac4-314f-addc-866190247439.test.csv\n",
      "Processed KPI 4d2af31a-9916-3d9f-8a8e-8a268a48c095 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-4d2af31a-9916-3d9f-8a8e-8a268a48c095.train.csv\n",
      "Processed KPI 4d2af31a-9916-3d9f-8a8e-8a268a48c095 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-4d2af31a-9916-3d9f-8a8e-8a268a48c095.test.csv\n",
      "Processed KPI 6a757df4-95e5-3357-8406-165e2bd49360 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-6a757df4-95e5-3357-8406-165e2bd49360.train.csv\n",
      "Processed KPI 6a757df4-95e5-3357-8406-165e2bd49360 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-6a757df4-95e5-3357-8406-165e2bd49360.test.csv\n",
      "Processed KPI ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa.train.csv\n",
      "Processed KPI ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa.test.csv\n",
      "Processed KPI 57051487-3a40-3828-9084-a12f7f23ee38 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-57051487-3a40-3828-9084-a12f7f23ee38.train.csv\n",
      "Processed KPI 57051487-3a40-3828-9084-a12f7f23ee38 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-57051487-3a40-3828-9084-a12f7f23ee38.test.csv\n",
      "Processed KPI f0932edd-6400-3e63-9559-0a9860a1baa9 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-f0932edd-6400-3e63-9559-0a9860a1baa9.train.csv\n",
      "Processed KPI f0932edd-6400-3e63-9559-0a9860a1baa9 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-f0932edd-6400-3e63-9559-0a9860a1baa9.test.csv\n",
      "Processed KPI 431a8542-c468-3988-a508-3afd06a218da -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-431a8542-c468-3988-a508-3afd06a218da.train.csv\n",
      "Processed KPI 431a8542-c468-3988-a508-3afd06a218da -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-431a8542-c468-3988-a508-3afd06a218da.test.csv\n",
      "Processed KPI 1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0.train.csv\n",
      "Processed KPI 1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0.test.csv\n",
      "Processed KPI c69a50cf-ee03-3bd7-831e-407d36c7ee91 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-c69a50cf-ee03-3bd7-831e-407d36c7ee91.train.csv\n",
      "Processed KPI c69a50cf-ee03-3bd7-831e-407d36c7ee91 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-c69a50cf-ee03-3bd7-831e-407d36c7ee91.test.csv\n",
      "Processed KPI 05f10d3a-239c-3bef-9bdc-a2feeb0037aa -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-05f10d3a-239c-3bef-9bdc-a2feeb0037aa.train.csv\n",
      "Processed KPI 05f10d3a-239c-3bef-9bdc-a2feeb0037aa -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-05f10d3a-239c-3bef-9bdc-a2feeb0037aa.test.csv\n",
      "Processed KPI 847e8ecc-f8d2-3a93-9107-f367a0aab37d -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-847e8ecc-f8d2-3a93-9107-f367a0aab37d.train.csv\n",
      "Processed KPI 847e8ecc-f8d2-3a93-9107-f367a0aab37d -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-847e8ecc-f8d2-3a93-9107-f367a0aab37d.test.csv\n",
      "Processed KPI 6efa3a07-4544-34a0-b921-a155bd1a05e8 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-6efa3a07-4544-34a0-b921-a155bd1a05e8.train.csv\n",
      "Processed KPI 6efa3a07-4544-34a0-b921-a155bd1a05e8 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-6efa3a07-4544-34a0-b921-a155bd1a05e8.test.csv\n",
      "Processed KPI 43115f2a-baeb-3b01-96f7-4ea14188343c -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-43115f2a-baeb-3b01-96f7-4ea14188343c.train.csv\n",
      "Processed KPI 43115f2a-baeb-3b01-96f7-4ea14188343c -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-43115f2a-baeb-3b01-96f7-4ea14188343c.test.csv\n",
      "Processed KPI 9c639a46-34c8-39bc-aaf0-9144b37adfc8 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-9c639a46-34c8-39bc-aaf0-9144b37adfc8.train.csv\n",
      "Processed KPI 9c639a46-34c8-39bc-aaf0-9144b37adfc8 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-9c639a46-34c8-39bc-aaf0-9144b37adfc8.test.csv\n",
      "Processed KPI a07ac296-de40-3a7c-8df3-91f642cc14d0 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-a07ac296-de40-3a7c-8df3-91f642cc14d0.train.csv\n",
      "Processed KPI a07ac296-de40-3a7c-8df3-91f642cc14d0 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-a07ac296-de40-3a7c-8df3-91f642cc14d0.test.csv\n",
      "Processed KPI ba5f3328-9f3f-3ff5-a683-84437d16d554 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-ba5f3328-9f3f-3ff5-a683-84437d16d554.train.csv\n",
      "Processed KPI ba5f3328-9f3f-3ff5-a683-84437d16d554 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-ba5f3328-9f3f-3ff5-a683-84437d16d554.test.csv\n",
      "Processed KPI 55f8b8b8-b659-38df-b3df-e4a5a8a54bc9 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-55f8b8b8-b659-38df-b3df-e4a5a8a54bc9.train.csv\n",
      "Processed KPI 55f8b8b8-b659-38df-b3df-e4a5a8a54bc9 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-55f8b8b8-b659-38df-b3df-e4a5a8a54bc9.test.csv\n",
      "Processed KPI 6d1114ae-be04-3c46-b5aa-be1a003a57cd -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-6d1114ae-be04-3c46-b5aa-be1a003a57cd.train.csv\n",
      "Processed KPI 6d1114ae-be04-3c46-b5aa-be1a003a57cd -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-6d1114ae-be04-3c46-b5aa-be1a003a57cd.test.csv\n",
      "Processed KPI 8723f0fb-eaef-32e6-b372-6034c9c04b80 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-8723f0fb-eaef-32e6-b372-6034c9c04b80.train.csv\n",
      "Processed KPI 8723f0fb-eaef-32e6-b372-6034c9c04b80 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-8723f0fb-eaef-32e6-b372-6034c9c04b80.test.csv\n",
      "Processed KPI adb2fde9-8589-3f5b-a410-5fe14386c7af -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-adb2fde9-8589-3f5b-a410-5fe14386c7af.train.csv\n",
      "Processed KPI adb2fde9-8589-3f5b-a410-5fe14386c7af -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-adb2fde9-8589-3f5b-a410-5fe14386c7af.test.csv\n",
      "Processed KPI 42d6616d-c9c5-370a-a8ba-17ead74f3114 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-42d6616d-c9c5-370a-a8ba-17ead74f3114.train.csv\n",
      "Processed KPI 42d6616d-c9c5-370a-a8ba-17ead74f3114 -> /home/projects/akita/data/benchmark-data/data-processed/univariate/IOPS/KPI-42d6616d-c9c5-370a-a8ba-17ead74f3114.test.csv\n",
      "...done.\n",
      "Metadata saved.\n"
     ]
    }
   ],
   "source": [
    "print(\"Reading source datasets...\")\n",
    "df_train = pd.read_csv(os.path.join(source_folder, \"phase2_train.csv\"))\n",
    "df_test = pd.read_hdf(os.path.join(source_folder, \"phase2_ground_truth.hdf\"))\n",
    "df_test[\"KPI ID\"] = df_test[\"KPI ID\"].transform(lambda x: str(x))\n",
    "dfs = {\n",
    "    \"train\": df_train,\n",
    "    \"test\": df_test\n",
    "}\n",
    "print(\"...done.\\n\\nStarting processing...\")\n",
    "\n",
    "for kpi in df_test[\"KPI ID\"].unique():\n",
    "    dataset_length = 0\n",
    "    paths = {}\n",
    "\n",
    "    # transform file\n",
    "    for t_type in [\"train\", \"test\"]:\n",
    "        filename = f\"KPI-{kpi}.{t_type}.csv\"\n",
    "        path = os.path.join(dataset_subfolder, filename)\n",
    "        target_filepath = os.path.join(target_subfolder, filename)\n",
    "        paths[t_type] = path\n",
    "        df_target = transform(dfs[t_type], kpi)\n",
    "        if t_type == \"test\":\n",
    "            dataset_length = len(df_target)\n",
    "        df_target.to_csv(target_filepath, index=False)\n",
    "        print(f\"Processed KPI {kpi} -> {target_filepath}\")\n",
    "\n",
    "    dataset_name = kpi\n",
    "\n",
    "    # save metadata\n",
    "    dm.add_dataset((dataset_collection_name, dataset_name),\n",
    "        train_path = paths[\"train\"],\n",
    "        test_path = paths[\"test\"],\n",
    "        dataset_type = dataset_type,\n",
    "        datetime_index = datetime_index,\n",
    "        split_at = None,\n",
    "        train_type = train_type,\n",
    "        train_is_normal = train_is_normal,\n",
    "        input_type = input_type,\n",
    "        dataset_length = dataset_length\n",
    "    )\n",
    "print(\"...done.\")\n",
    "dm.save()\n",
    "print(\"Metadata saved.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>train_path</th>\n",
       "      <th>test_path</th>\n",
       "      <th>dataset_type</th>\n",
       "      <th>datetime_index</th>\n",
       "      <th>split_at</th>\n",
       "      <th>train_type</th>\n",
       "      <th>train_is_normal</th>\n",
       "      <th>input_type</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>collection_name</th>\n",
       "      <th>dataset_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"29\" valign=\"top\">IOPS</th>\n",
       "      <th>05f10d3a-239c-3bef-9bdc-a2feeb0037aa</th>\n",
       "      <td>univariate/IOPS/KPI-05f10d3a-239c-3bef-9bdc-a2...</td>\n",
       "      <td>univariate/IOPS/KPI-05f10d3a-239c-3bef-9bdc-a2...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0efb375b-b902-3661-ab23-9a0bb799f4e3</th>\n",
       "      <td>univariate/IOPS/KPI-0efb375b-b902-3661-ab23-9a...</td>\n",
       "      <td>univariate/IOPS/KPI-0efb375b-b902-3661-ab23-9a...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>8784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0</th>\n",
       "      <td>univariate/IOPS/KPI-1c6d7a26-1f1a-3321-bb4d-7a...</td>\n",
       "      <td>univariate/IOPS/KPI-1c6d7a26-1f1a-3321-bb4d-7a...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301c70d8-1630-35ac-8f96-bc1b6f4359ea</th>\n",
       "      <td>univariate/IOPS/KPI-301c70d8-1630-35ac-8f96-bc...</td>\n",
       "      <td>univariate/IOPS/KPI-301c70d8-1630-35ac-8f96-bc...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>8784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42d6616d-c9c5-370a-a8ba-17ead74f3114</th>\n",
       "      <td>univariate/IOPS/KPI-42d6616d-c9c5-370a-a8ba-17...</td>\n",
       "      <td>univariate/IOPS/KPI-42d6616d-c9c5-370a-a8ba-17...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43115f2a-baeb-3b01-96f7-4ea14188343c</th>\n",
       "      <td>univariate/IOPS/KPI-43115f2a-baeb-3b01-96f7-4e...</td>\n",
       "      <td>univariate/IOPS/KPI-43115f2a-baeb-3b01-96f7-4e...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>110629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431a8542-c468-3988-a508-3afd06a218da</th>\n",
       "      <td>univariate/IOPS/KPI-431a8542-c468-3988-a508-3a...</td>\n",
       "      <td>univariate/IOPS/KPI-431a8542-c468-3988-a508-3a...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>111566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4d2af31a-9916-3d9f-8a8e-8a268a48c095</th>\n",
       "      <td>univariate/IOPS/KPI-4d2af31a-9916-3d9f-8a8e-8a...</td>\n",
       "      <td>univariate/IOPS/KPI-4d2af31a-9916-3d9f-8a8e-8a...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>111370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54350a12-7a9d-3ca8-b81f-f886b9d156fd</th>\n",
       "      <td>univariate/IOPS/KPI-54350a12-7a9d-3ca8-b81f-f8...</td>\n",
       "      <td>univariate/IOPS/KPI-54350a12-7a9d-3ca8-b81f-f8...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>7616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55f8b8b8-b659-38df-b3df-e4a5a8a54bc9</th>\n",
       "      <td>univariate/IOPS/KPI-55f8b8b8-b659-38df-b3df-e4...</td>\n",
       "      <td>univariate/IOPS/KPI-55f8b8b8-b659-38df-b3df-e4...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57051487-3a40-3828-9084-a12f7f23ee38</th>\n",
       "      <td>univariate/IOPS/KPI-57051487-3a40-3828-9084-a1...</td>\n",
       "      <td>univariate/IOPS/KPI-57051487-3a40-3828-9084-a1...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>111679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6a757df4-95e5-3357-8406-165e2bd49360</th>\n",
       "      <td>univariate/IOPS/KPI-6a757df4-95e5-3357-8406-16...</td>\n",
       "      <td>univariate/IOPS/KPI-6a757df4-95e5-3357-8406-16...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>110876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6d1114ae-be04-3c46-b5aa-be1a003a57cd</th>\n",
       "      <td>univariate/IOPS/KPI-6d1114ae-be04-3c46-b5aa-be...</td>\n",
       "      <td>univariate/IOPS/KPI-6d1114ae-be04-3c46-b5aa-be...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6efa3a07-4544-34a0-b921-a155bd1a05e8</th>\n",
       "      <td>univariate/IOPS/KPI-6efa3a07-4544-34a0-b921-a1...</td>\n",
       "      <td>univariate/IOPS/KPI-6efa3a07-4544-34a0-b921-a1...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7103fa0f-cac4-314f-addc-866190247439</th>\n",
       "      <td>univariate/IOPS/KPI-7103fa0f-cac4-314f-addc-86...</td>\n",
       "      <td>univariate/IOPS/KPI-7103fa0f-cac4-314f-addc-86...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>108425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847e8ecc-f8d2-3a93-9107-f367a0aab37d</th>\n",
       "      <td>univariate/IOPS/KPI-847e8ecc-f8d2-3a93-9107-f3...</td>\n",
       "      <td>univariate/IOPS/KPI-847e8ecc-f8d2-3a93-9107-f3...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8723f0fb-eaef-32e6-b372-6034c9c04b80</th>\n",
       "      <td>univariate/IOPS/KPI-8723f0fb-eaef-32e6-b372-60...</td>\n",
       "      <td>univariate/IOPS/KPI-8723f0fb-eaef-32e6-b372-60...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9c639a46-34c8-39bc-aaf0-9144b37adfc8</th>\n",
       "      <td>univariate/IOPS/KPI-9c639a46-34c8-39bc-aaf0-91...</td>\n",
       "      <td>univariate/IOPS/KPI-9c639a46-34c8-39bc-aaf0-91...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>110636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a07ac296-de40-3a7c-8df3-91f642cc14d0</th>\n",
       "      <td>univariate/IOPS/KPI-a07ac296-de40-3a7c-8df3-91...</td>\n",
       "      <td>univariate/IOPS/KPI-a07ac296-de40-3a7c-8df3-91...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>111307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a8c06b47-cc41-3738-9110-12df0ee4c721</th>\n",
       "      <td>univariate/IOPS/KPI-a8c06b47-cc41-3738-9110-12...</td>\n",
       "      <td>univariate/IOPS/KPI-a8c06b47-cc41-3738-9110-12...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>7578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ab216663-dcc2-3a24-b1ee-2c3e550e06c9</th>\n",
       "      <td>univariate/IOPS/KPI-ab216663-dcc2-3a24-b1ee-2c...</td>\n",
       "      <td>univariate/IOPS/KPI-ab216663-dcc2-3a24-b1ee-2c...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>10780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adb2fde9-8589-3f5b-a410-5fe14386c7af</th>\n",
       "      <td>univariate/IOPS/KPI-adb2fde9-8589-3f5b-a410-5f...</td>\n",
       "      <td>univariate/IOPS/KPI-adb2fde9-8589-3f5b-a410-5f...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ba5f3328-9f3f-3ff5-a683-84437d16d554</th>\n",
       "      <td>univariate/IOPS/KPI-ba5f3328-9f3f-3ff5-a683-84...</td>\n",
       "      <td>univariate/IOPS/KPI-ba5f3328-9f3f-3ff5-a683-84...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c02607e8-7399-3dde-9d28-8a8da5e5d251</th>\n",
       "      <td>univariate/IOPS/KPI-c02607e8-7399-3dde-9d28-8a...</td>\n",
       "      <td>univariate/IOPS/KPI-c02607e8-7399-3dde-9d28-8a...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>8784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c69a50cf-ee03-3bd7-831e-407d36c7ee91</th>\n",
       "      <td>univariate/IOPS/KPI-c69a50cf-ee03-3bd7-831e-40...</td>\n",
       "      <td>univariate/IOPS/KPI-c69a50cf-ee03-3bd7-831e-40...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>149159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>da10a69f-d836-3baa-ad40-3e548ecf1fbd</th>\n",
       "      <td>univariate/IOPS/KPI-da10a69f-d836-3baa-ad40-3e...</td>\n",
       "      <td>univariate/IOPS/KPI-da10a69f-d836-3baa-ad40-3e...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>107167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e0747cad-8dc8-38a9-a9ab-855b61f5551d</th>\n",
       "      <td>univariate/IOPS/KPI-e0747cad-8dc8-38a9-a9ab-85...</td>\n",
       "      <td>univariate/IOPS/KPI-e0747cad-8dc8-38a9-a9ab-85...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>8784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f0932edd-6400-3e63-9559-0a9860a1baa9</th>\n",
       "      <td>univariate/IOPS/KPI-f0932edd-6400-3e63-9559-0a...</td>\n",
       "      <td>univariate/IOPS/KPI-f0932edd-6400-3e63-9559-0a...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>112149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa</th>\n",
       "      <td>univariate/IOPS/KPI-ffb82d38-5f00-37db-abc0-5d...</td>\n",
       "      <td>univariate/IOPS/KPI-ffb82d38-5f00-37db-abc0-5d...</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>111396</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                             train_path  \\\n",
       "collection_name dataset_name                                                                              \n",
       "IOPS            05f10d3a-239c-3bef-9bdc-a2feeb0037aa  univariate/IOPS/KPI-05f10d3a-239c-3bef-9bdc-a2...   \n",
       "                0efb375b-b902-3661-ab23-9a0bb799f4e3  univariate/IOPS/KPI-0efb375b-b902-3661-ab23-9a...   \n",
       "                1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0  univariate/IOPS/KPI-1c6d7a26-1f1a-3321-bb4d-7a...   \n",
       "                301c70d8-1630-35ac-8f96-bc1b6f4359ea  univariate/IOPS/KPI-301c70d8-1630-35ac-8f96-bc...   \n",
       "                42d6616d-c9c5-370a-a8ba-17ead74f3114  univariate/IOPS/KPI-42d6616d-c9c5-370a-a8ba-17...   \n",
       "                43115f2a-baeb-3b01-96f7-4ea14188343c  univariate/IOPS/KPI-43115f2a-baeb-3b01-96f7-4e...   \n",
       "                431a8542-c468-3988-a508-3afd06a218da  univariate/IOPS/KPI-431a8542-c468-3988-a508-3a...   \n",
       "                4d2af31a-9916-3d9f-8a8e-8a268a48c095  univariate/IOPS/KPI-4d2af31a-9916-3d9f-8a8e-8a...   \n",
       "                54350a12-7a9d-3ca8-b81f-f886b9d156fd  univariate/IOPS/KPI-54350a12-7a9d-3ca8-b81f-f8...   \n",
       "                55f8b8b8-b659-38df-b3df-e4a5a8a54bc9  univariate/IOPS/KPI-55f8b8b8-b659-38df-b3df-e4...   \n",
       "                57051487-3a40-3828-9084-a12f7f23ee38  univariate/IOPS/KPI-57051487-3a40-3828-9084-a1...   \n",
       "                6a757df4-95e5-3357-8406-165e2bd49360  univariate/IOPS/KPI-6a757df4-95e5-3357-8406-16...   \n",
       "                6d1114ae-be04-3c46-b5aa-be1a003a57cd  univariate/IOPS/KPI-6d1114ae-be04-3c46-b5aa-be...   \n",
       "                6efa3a07-4544-34a0-b921-a155bd1a05e8  univariate/IOPS/KPI-6efa3a07-4544-34a0-b921-a1...   \n",
       "                7103fa0f-cac4-314f-addc-866190247439  univariate/IOPS/KPI-7103fa0f-cac4-314f-addc-86...   \n",
       "                847e8ecc-f8d2-3a93-9107-f367a0aab37d  univariate/IOPS/KPI-847e8ecc-f8d2-3a93-9107-f3...   \n",
       "                8723f0fb-eaef-32e6-b372-6034c9c04b80  univariate/IOPS/KPI-8723f0fb-eaef-32e6-b372-60...   \n",
       "                9c639a46-34c8-39bc-aaf0-9144b37adfc8  univariate/IOPS/KPI-9c639a46-34c8-39bc-aaf0-91...   \n",
       "                a07ac296-de40-3a7c-8df3-91f642cc14d0  univariate/IOPS/KPI-a07ac296-de40-3a7c-8df3-91...   \n",
       "                a8c06b47-cc41-3738-9110-12df0ee4c721  univariate/IOPS/KPI-a8c06b47-cc41-3738-9110-12...   \n",
       "                ab216663-dcc2-3a24-b1ee-2c3e550e06c9  univariate/IOPS/KPI-ab216663-dcc2-3a24-b1ee-2c...   \n",
       "                adb2fde9-8589-3f5b-a410-5fe14386c7af  univariate/IOPS/KPI-adb2fde9-8589-3f5b-a410-5f...   \n",
       "                ba5f3328-9f3f-3ff5-a683-84437d16d554  univariate/IOPS/KPI-ba5f3328-9f3f-3ff5-a683-84...   \n",
       "                c02607e8-7399-3dde-9d28-8a8da5e5d251  univariate/IOPS/KPI-c02607e8-7399-3dde-9d28-8a...   \n",
       "                c69a50cf-ee03-3bd7-831e-407d36c7ee91  univariate/IOPS/KPI-c69a50cf-ee03-3bd7-831e-40...   \n",
       "                da10a69f-d836-3baa-ad40-3e548ecf1fbd  univariate/IOPS/KPI-da10a69f-d836-3baa-ad40-3e...   \n",
       "                e0747cad-8dc8-38a9-a9ab-855b61f5551d  univariate/IOPS/KPI-e0747cad-8dc8-38a9-a9ab-85...   \n",
       "                f0932edd-6400-3e63-9559-0a9860a1baa9  univariate/IOPS/KPI-f0932edd-6400-3e63-9559-0a...   \n",
       "                ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa  univariate/IOPS/KPI-ffb82d38-5f00-37db-abc0-5d...   \n",
       "\n",
       "                                                                                              test_path  \\\n",
       "collection_name dataset_name                                                                              \n",
       "IOPS            05f10d3a-239c-3bef-9bdc-a2feeb0037aa  univariate/IOPS/KPI-05f10d3a-239c-3bef-9bdc-a2...   \n",
       "                0efb375b-b902-3661-ab23-9a0bb799f4e3  univariate/IOPS/KPI-0efb375b-b902-3661-ab23-9a...   \n",
       "                1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0  univariate/IOPS/KPI-1c6d7a26-1f1a-3321-bb4d-7a...   \n",
       "                301c70d8-1630-35ac-8f96-bc1b6f4359ea  univariate/IOPS/KPI-301c70d8-1630-35ac-8f96-bc...   \n",
       "                42d6616d-c9c5-370a-a8ba-17ead74f3114  univariate/IOPS/KPI-42d6616d-c9c5-370a-a8ba-17...   \n",
       "                43115f2a-baeb-3b01-96f7-4ea14188343c  univariate/IOPS/KPI-43115f2a-baeb-3b01-96f7-4e...   \n",
       "                431a8542-c468-3988-a508-3afd06a218da  univariate/IOPS/KPI-431a8542-c468-3988-a508-3a...   \n",
       "                4d2af31a-9916-3d9f-8a8e-8a268a48c095  univariate/IOPS/KPI-4d2af31a-9916-3d9f-8a8e-8a...   \n",
       "                54350a12-7a9d-3ca8-b81f-f886b9d156fd  univariate/IOPS/KPI-54350a12-7a9d-3ca8-b81f-f8...   \n",
       "                55f8b8b8-b659-38df-b3df-e4a5a8a54bc9  univariate/IOPS/KPI-55f8b8b8-b659-38df-b3df-e4...   \n",
       "                57051487-3a40-3828-9084-a12f7f23ee38  univariate/IOPS/KPI-57051487-3a40-3828-9084-a1...   \n",
       "                6a757df4-95e5-3357-8406-165e2bd49360  univariate/IOPS/KPI-6a757df4-95e5-3357-8406-16...   \n",
       "                6d1114ae-be04-3c46-b5aa-be1a003a57cd  univariate/IOPS/KPI-6d1114ae-be04-3c46-b5aa-be...   \n",
       "                6efa3a07-4544-34a0-b921-a155bd1a05e8  univariate/IOPS/KPI-6efa3a07-4544-34a0-b921-a1...   \n",
       "                7103fa0f-cac4-314f-addc-866190247439  univariate/IOPS/KPI-7103fa0f-cac4-314f-addc-86...   \n",
       "                847e8ecc-f8d2-3a93-9107-f367a0aab37d  univariate/IOPS/KPI-847e8ecc-f8d2-3a93-9107-f3...   \n",
       "                8723f0fb-eaef-32e6-b372-6034c9c04b80  univariate/IOPS/KPI-8723f0fb-eaef-32e6-b372-60...   \n",
       "                9c639a46-34c8-39bc-aaf0-9144b37adfc8  univariate/IOPS/KPI-9c639a46-34c8-39bc-aaf0-91...   \n",
       "                a07ac296-de40-3a7c-8df3-91f642cc14d0  univariate/IOPS/KPI-a07ac296-de40-3a7c-8df3-91...   \n",
       "                a8c06b47-cc41-3738-9110-12df0ee4c721  univariate/IOPS/KPI-a8c06b47-cc41-3738-9110-12...   \n",
       "                ab216663-dcc2-3a24-b1ee-2c3e550e06c9  univariate/IOPS/KPI-ab216663-dcc2-3a24-b1ee-2c...   \n",
       "                adb2fde9-8589-3f5b-a410-5fe14386c7af  univariate/IOPS/KPI-adb2fde9-8589-3f5b-a410-5f...   \n",
       "                ba5f3328-9f3f-3ff5-a683-84437d16d554  univariate/IOPS/KPI-ba5f3328-9f3f-3ff5-a683-84...   \n",
       "                c02607e8-7399-3dde-9d28-8a8da5e5d251  univariate/IOPS/KPI-c02607e8-7399-3dde-9d28-8a...   \n",
       "                c69a50cf-ee03-3bd7-831e-407d36c7ee91  univariate/IOPS/KPI-c69a50cf-ee03-3bd7-831e-40...   \n",
       "                da10a69f-d836-3baa-ad40-3e548ecf1fbd  univariate/IOPS/KPI-da10a69f-d836-3baa-ad40-3e...   \n",
       "                e0747cad-8dc8-38a9-a9ab-855b61f5551d  univariate/IOPS/KPI-e0747cad-8dc8-38a9-a9ab-85...   \n",
       "                f0932edd-6400-3e63-9559-0a9860a1baa9  univariate/IOPS/KPI-f0932edd-6400-3e63-9559-0a...   \n",
       "                ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa  univariate/IOPS/KPI-ffb82d38-5f00-37db-abc0-5d...   \n",
       "\n",
       "                                                     dataset_type  \\\n",
       "collection_name dataset_name                                        \n",
       "IOPS            05f10d3a-239c-3bef-9bdc-a2feeb0037aa         real   \n",
       "                0efb375b-b902-3661-ab23-9a0bb799f4e3         real   \n",
       "                1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0         real   \n",
       "                301c70d8-1630-35ac-8f96-bc1b6f4359ea         real   \n",
       "                42d6616d-c9c5-370a-a8ba-17ead74f3114         real   \n",
       "                43115f2a-baeb-3b01-96f7-4ea14188343c         real   \n",
       "                431a8542-c468-3988-a508-3afd06a218da         real   \n",
       "                4d2af31a-9916-3d9f-8a8e-8a268a48c095         real   \n",
       "                54350a12-7a9d-3ca8-b81f-f886b9d156fd         real   \n",
       "                55f8b8b8-b659-38df-b3df-e4a5a8a54bc9         real   \n",
       "                57051487-3a40-3828-9084-a12f7f23ee38         real   \n",
       "                6a757df4-95e5-3357-8406-165e2bd49360         real   \n",
       "                6d1114ae-be04-3c46-b5aa-be1a003a57cd         real   \n",
       "                6efa3a07-4544-34a0-b921-a155bd1a05e8         real   \n",
       "                7103fa0f-cac4-314f-addc-866190247439         real   \n",
       "                847e8ecc-f8d2-3a93-9107-f367a0aab37d         real   \n",
       "                8723f0fb-eaef-32e6-b372-6034c9c04b80         real   \n",
       "                9c639a46-34c8-39bc-aaf0-9144b37adfc8         real   \n",
       "                a07ac296-de40-3a7c-8df3-91f642cc14d0         real   \n",
       "                a8c06b47-cc41-3738-9110-12df0ee4c721         real   \n",
       "                ab216663-dcc2-3a24-b1ee-2c3e550e06c9         real   \n",
       "                adb2fde9-8589-3f5b-a410-5fe14386c7af         real   \n",
       "                ba5f3328-9f3f-3ff5-a683-84437d16d554         real   \n",
       "                c02607e8-7399-3dde-9d28-8a8da5e5d251         real   \n",
       "                c69a50cf-ee03-3bd7-831e-407d36c7ee91         real   \n",
       "                da10a69f-d836-3baa-ad40-3e548ecf1fbd         real   \n",
       "                e0747cad-8dc8-38a9-a9ab-855b61f5551d         real   \n",
       "                f0932edd-6400-3e63-9559-0a9860a1baa9         real   \n",
       "                ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa         real   \n",
       "\n",
       "                                                      datetime_index  \\\n",
       "collection_name dataset_name                                           \n",
       "IOPS            05f10d3a-239c-3bef-9bdc-a2feeb0037aa            True   \n",
       "                0efb375b-b902-3661-ab23-9a0bb799f4e3            True   \n",
       "                1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0            True   \n",
       "                301c70d8-1630-35ac-8f96-bc1b6f4359ea            True   \n",
       "                42d6616d-c9c5-370a-a8ba-17ead74f3114            True   \n",
       "                43115f2a-baeb-3b01-96f7-4ea14188343c            True   \n",
       "                431a8542-c468-3988-a508-3afd06a218da            True   \n",
       "                4d2af31a-9916-3d9f-8a8e-8a268a48c095            True   \n",
       "                54350a12-7a9d-3ca8-b81f-f886b9d156fd            True   \n",
       "                55f8b8b8-b659-38df-b3df-e4a5a8a54bc9            True   \n",
       "                57051487-3a40-3828-9084-a12f7f23ee38            True   \n",
       "                6a757df4-95e5-3357-8406-165e2bd49360            True   \n",
       "                6d1114ae-be04-3c46-b5aa-be1a003a57cd            True   \n",
       "                6efa3a07-4544-34a0-b921-a155bd1a05e8            True   \n",
       "                7103fa0f-cac4-314f-addc-866190247439            True   \n",
       "                847e8ecc-f8d2-3a93-9107-f367a0aab37d            True   \n",
       "                8723f0fb-eaef-32e6-b372-6034c9c04b80            True   \n",
       "                9c639a46-34c8-39bc-aaf0-9144b37adfc8            True   \n",
       "                a07ac296-de40-3a7c-8df3-91f642cc14d0            True   \n",
       "                a8c06b47-cc41-3738-9110-12df0ee4c721            True   \n",
       "                ab216663-dcc2-3a24-b1ee-2c3e550e06c9            True   \n",
       "                adb2fde9-8589-3f5b-a410-5fe14386c7af            True   \n",
       "                ba5f3328-9f3f-3ff5-a683-84437d16d554            True   \n",
       "                c02607e8-7399-3dde-9d28-8a8da5e5d251            True   \n",
       "                c69a50cf-ee03-3bd7-831e-407d36c7ee91            True   \n",
       "                da10a69f-d836-3baa-ad40-3e548ecf1fbd            True   \n",
       "                e0747cad-8dc8-38a9-a9ab-855b61f5551d            True   \n",
       "                f0932edd-6400-3e63-9559-0a9860a1baa9            True   \n",
       "                ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa            True   \n",
       "\n",
       "                                                      split_at  train_type  \\\n",
       "collection_name dataset_name                                                 \n",
       "IOPS            05f10d3a-239c-3bef-9bdc-a2feeb0037aa       NaN  supervised   \n",
       "                0efb375b-b902-3661-ab23-9a0bb799f4e3       NaN  supervised   \n",
       "                1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0       NaN  supervised   \n",
       "                301c70d8-1630-35ac-8f96-bc1b6f4359ea       NaN  supervised   \n",
       "                42d6616d-c9c5-370a-a8ba-17ead74f3114       NaN  supervised   \n",
       "                43115f2a-baeb-3b01-96f7-4ea14188343c       NaN  supervised   \n",
       "                431a8542-c468-3988-a508-3afd06a218da       NaN  supervised   \n",
       "                4d2af31a-9916-3d9f-8a8e-8a268a48c095       NaN  supervised   \n",
       "                54350a12-7a9d-3ca8-b81f-f886b9d156fd       NaN  supervised   \n",
       "                55f8b8b8-b659-38df-b3df-e4a5a8a54bc9       NaN  supervised   \n",
       "                57051487-3a40-3828-9084-a12f7f23ee38       NaN  supervised   \n",
       "                6a757df4-95e5-3357-8406-165e2bd49360       NaN  supervised   \n",
       "                6d1114ae-be04-3c46-b5aa-be1a003a57cd       NaN  supervised   \n",
       "                6efa3a07-4544-34a0-b921-a155bd1a05e8       NaN  supervised   \n",
       "                7103fa0f-cac4-314f-addc-866190247439       NaN  supervised   \n",
       "                847e8ecc-f8d2-3a93-9107-f367a0aab37d       NaN  supervised   \n",
       "                8723f0fb-eaef-32e6-b372-6034c9c04b80       NaN  supervised   \n",
       "                9c639a46-34c8-39bc-aaf0-9144b37adfc8       NaN  supervised   \n",
       "                a07ac296-de40-3a7c-8df3-91f642cc14d0       NaN  supervised   \n",
       "                a8c06b47-cc41-3738-9110-12df0ee4c721       NaN  supervised   \n",
       "                ab216663-dcc2-3a24-b1ee-2c3e550e06c9       NaN  supervised   \n",
       "                adb2fde9-8589-3f5b-a410-5fe14386c7af       NaN  supervised   \n",
       "                ba5f3328-9f3f-3ff5-a683-84437d16d554       NaN  supervised   \n",
       "                c02607e8-7399-3dde-9d28-8a8da5e5d251       NaN  supervised   \n",
       "                c69a50cf-ee03-3bd7-831e-407d36c7ee91       NaN  supervised   \n",
       "                da10a69f-d836-3baa-ad40-3e548ecf1fbd       NaN  supervised   \n",
       "                e0747cad-8dc8-38a9-a9ab-855b61f5551d       NaN  supervised   \n",
       "                f0932edd-6400-3e63-9559-0a9860a1baa9       NaN  supervised   \n",
       "                ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa       NaN  supervised   \n",
       "\n",
       "                                                      train_is_normal  \\\n",
       "collection_name dataset_name                                            \n",
       "IOPS            05f10d3a-239c-3bef-9bdc-a2feeb0037aa            False   \n",
       "                0efb375b-b902-3661-ab23-9a0bb799f4e3            False   \n",
       "                1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0            False   \n",
       "                301c70d8-1630-35ac-8f96-bc1b6f4359ea            False   \n",
       "                42d6616d-c9c5-370a-a8ba-17ead74f3114            False   \n",
       "                43115f2a-baeb-3b01-96f7-4ea14188343c            False   \n",
       "                431a8542-c468-3988-a508-3afd06a218da            False   \n",
       "                4d2af31a-9916-3d9f-8a8e-8a268a48c095            False   \n",
       "                54350a12-7a9d-3ca8-b81f-f886b9d156fd            False   \n",
       "                55f8b8b8-b659-38df-b3df-e4a5a8a54bc9            False   \n",
       "                57051487-3a40-3828-9084-a12f7f23ee38            False   \n",
       "                6a757df4-95e5-3357-8406-165e2bd49360            False   \n",
       "                6d1114ae-be04-3c46-b5aa-be1a003a57cd            False   \n",
       "                6efa3a07-4544-34a0-b921-a155bd1a05e8            False   \n",
       "                7103fa0f-cac4-314f-addc-866190247439            False   \n",
       "                847e8ecc-f8d2-3a93-9107-f367a0aab37d            False   \n",
       "                8723f0fb-eaef-32e6-b372-6034c9c04b80            False   \n",
       "                9c639a46-34c8-39bc-aaf0-9144b37adfc8            False   \n",
       "                a07ac296-de40-3a7c-8df3-91f642cc14d0            False   \n",
       "                a8c06b47-cc41-3738-9110-12df0ee4c721            False   \n",
       "                ab216663-dcc2-3a24-b1ee-2c3e550e06c9            False   \n",
       "                adb2fde9-8589-3f5b-a410-5fe14386c7af            False   \n",
       "                ba5f3328-9f3f-3ff5-a683-84437d16d554            False   \n",
       "                c02607e8-7399-3dde-9d28-8a8da5e5d251            False   \n",
       "                c69a50cf-ee03-3bd7-831e-407d36c7ee91            False   \n",
       "                da10a69f-d836-3baa-ad40-3e548ecf1fbd            False   \n",
       "                e0747cad-8dc8-38a9-a9ab-855b61f5551d            False   \n",
       "                f0932edd-6400-3e63-9559-0a9860a1baa9            False   \n",
       "                ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa            False   \n",
       "\n",
       "                                                      input_type  length  \n",
       "collection_name dataset_name                                              \n",
       "IOPS            05f10d3a-239c-3bef-9bdc-a2feeb0037aa  univariate  149130  \n",
       "                0efb375b-b902-3661-ab23-9a0bb799f4e3  univariate    8784  \n",
       "                1c6d7a26-1f1a-3321-bb4d-7a9d969ec8f0  univariate  149156  \n",
       "                301c70d8-1630-35ac-8f96-bc1b6f4359ea  univariate    8784  \n",
       "                42d6616d-c9c5-370a-a8ba-17ead74f3114  univariate  149161  \n",
       "                43115f2a-baeb-3b01-96f7-4ea14188343c  univariate  110629  \n",
       "                431a8542-c468-3988-a508-3afd06a218da  univariate  111566  \n",
       "                4d2af31a-9916-3d9f-8a8e-8a268a48c095  univariate  111370  \n",
       "                54350a12-7a9d-3ca8-b81f-f886b9d156fd  univariate    7616  \n",
       "                55f8b8b8-b659-38df-b3df-e4a5a8a54bc9  univariate  149133  \n",
       "                57051487-3a40-3828-9084-a12f7f23ee38  univariate  111679  \n",
       "                6a757df4-95e5-3357-8406-165e2bd49360  univariate  110876  \n",
       "                6d1114ae-be04-3c46-b5aa-be1a003a57cd  univariate  149122  \n",
       "                6efa3a07-4544-34a0-b921-a155bd1a05e8  univariate  149148  \n",
       "                7103fa0f-cac4-314f-addc-866190247439  univariate  108425  \n",
       "                847e8ecc-f8d2-3a93-9107-f367a0aab37d  univariate  149123  \n",
       "                8723f0fb-eaef-32e6-b372-6034c9c04b80  univariate  149118  \n",
       "                9c639a46-34c8-39bc-aaf0-9144b37adfc8  univariate  110636  \n",
       "                a07ac296-de40-3a7c-8df3-91f642cc14d0  univariate  111307  \n",
       "                a8c06b47-cc41-3738-9110-12df0ee4c721  univariate    7578  \n",
       "                ab216663-dcc2-3a24-b1ee-2c3e550e06c9  univariate   10780  \n",
       "                adb2fde9-8589-3f5b-a410-5fe14386c7af  univariate  149155  \n",
       "                ba5f3328-9f3f-3ff5-a683-84437d16d554  univariate  149132  \n",
       "                c02607e8-7399-3dde-9d28-8a8da5e5d251  univariate    8784  \n",
       "                c69a50cf-ee03-3bd7-831e-407d36c7ee91  univariate  149159  \n",
       "                da10a69f-d836-3baa-ad40-3e548ecf1fbd  univariate  107167  \n",
       "                e0747cad-8dc8-38a9-a9ab-855b61f5551d  univariate    8784  \n",
       "                f0932edd-6400-3e63-9559-0a9860a1baa9  univariate  112149  \n",
       "                ffb82d38-5f00-37db-abc0-5d2e4e4cb6aa  univariate  111396  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dm.refresh()\n",
    "dm.df().loc[(slice(dataset_collection_name,dataset_collection_name), slice(None))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experimentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.listdir(source_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data-raw/IOPS AI Challenge/KPI Anomaly Detection Final Data Set/phase2_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_hdf(\"data-raw/IOPS AI Challenge/KPI Anomaly Detection Final Data Set/phase2_ground_truth.hdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[\"KPI ID\"] = test[\"KPI ID\"].transform(lambda x: str(x))\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[\"KPI ID\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform(train, \"da10a69f-d836-3baa-ad40-3e548ecf1fbd\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform(test, \"da10a69f-d836-3baa-ad40-3e548ecf1fbd\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dfs = [x for _, x in train.groupby(\"KPI ID\")]\n",
    "train_dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dfs = [x for _, x in test.groupby(\"KPI ID\")]\n",
    "test_dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_kpis = train[\"KPI ID\"].unique()\n",
    "test_kpis = test[\"KPI ID\"].unique()\n",
    "set(test) - set(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in test_dfs:\n",
    "    df.plot(x=\"timestamp\", y=[\"value\"])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "timeeval",
   "language": "python",
   "name": "timeeval"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
